#!/usr/bin/env python
# -*- coding:utf-8 -*-
# ---
# @Time    : 2018/12/6 19:24
# @Author  : liujiantao
# @Site    :
# @File    :
from tiancheng.base.base_helper import *

y = get_tag_train_new()
sub = get_sub()

op_train = get_operation_train_new()
op_test = get_operation_round1_new()
op_train = op_train.merge(y, on=tag_hd.UID, how='left')
op_test = op_test.merge(y, on=tag_hd.UID, how='left')
op_train, op_test = deal_data_v1(op_train, op_test)

trans_train = get_transaction_train_new()
trans_test = get_transaction_round1_new()
trans_train = trans_train.merge(y, on=tag_hd.UID, how='left')
trans_test = trans_test.merge(y, on=tag_hd.UID, how='left')
trans_train, trans_test = deal_data_v1(trans_train, trans_test)
op_train = fill_mean(op_train)
op_test = fill_mean(op_test)
trans_train = fill_mean(trans_train)
trans_test = fill_mean(trans_test)


def get_feature(op, trans, label):
    try:
        op.pop(tag_hd.Tag)
        trans.pop(tag_hd.Tag)
    except:
        pass
    try:
        for feature in op.columns[1:]:
            print(feature)
            if feature not in ['day']:
                if feature != 'UID':
                    label = label.merge(op.groupby(['UID'])[feature].count().reset_index(), on='UID', how='left')
                    label = label.merge(op.groupby(['UID'])[feature].nunique().reset_index(), on='UID', how='left')
                for deliver in ['ip1', 'mac1', 'mac2', 'geo_code']:
                    if feature not in deliver:
                        if feature != 'UID':
                            temp = \
                                op[['UID', deliver]].merge(op.groupby([deliver])[feature].count().reset_index(),
                                                           on=deliver,
                                                           how='left')[['UID', feature]]
                            temp = temp.groupby('UID')[feature].sum().reset_index()
                            temp.columns = ['UID', feature + deliver]
                            label = label.merge(temp, on='UID', how='left')
                            del temp
                            temp = \
                                op[['UID', deliver]].merge(op.groupby([deliver])[feature].nunique().reset_index(),
                                                           on=deliver,
                                                           how='left')[['UID', feature]]
                            temp = temp.groupby('UID')[feature].sum().reset_index()
                            temp.columns = ['UID', feature + deliver]
                            label = label.merge(temp, on='UID', how='left')
                            del temp
                        else:
                            temp = \
                                op[['UID', deliver]].merge(op.groupby([deliver])[feature].count().reset_index(),
                                                           on=deliver,
                                                           how='left')[['UID_x', 'UID_y']]
                            temp = temp.groupby('UID_x')['UID_y'].sum().reset_index()
                            temp.columns = ['UID', feature + deliver]
                            label = label.merge(temp, on='UID', how='left')
                            del temp
                            temp = \
                                op[['UID', deliver]].merge(op.groupby([deliver])[feature].nunique().reset_index(),
                                                           on=deliver,
                                                           how='left')[['UID_x', 'UID_y']]
                            temp = temp.groupby('UID_x')['UID_y'].sum().reset_index()
                            temp.columns = ['UID', feature + deliver]
                            label = label.merge(temp, on='UID', how='left')
                            del temp

            else:
                print(feature)
                label = label.merge(op.groupby(['UID'])[feature].count().reset_index(), on='UID', how='left')
                label = label.merge(op.groupby(['UID'])[feature].nunique().reset_index(), on='UID', how='left')
                label = label.merge(op.groupby(['UID'])[feature].max().reset_index(), on='UID', how='left')
                label = label.merge(op.groupby(['UID'])[feature].min().reset_index(), on='UID', how='left')
                label = label.merge(op.groupby(['UID'])[feature].sum().reset_index(), on='UID', how='left')
                label = label.merge(op.groupby(['UID'])[feature].mean().reset_index(), on='UID', how='left')
                label = label.merge(op.groupby(['UID'])[feature].std().reset_index(), on='UID', how='left')
                for deliver in ['ip1', 'mac1', 'mac2']:
                    if feature not in deliver:
                        temp = \
                            op[['UID', deliver]].merge(op.groupby([deliver])[feature].count().reset_index(), on=deliver,
                                                       how='left')[['UID', feature]]
                        temp = temp.groupby('UID')[feature].sum().reset_index()
                        temp.columns = ['UID', feature + deliver]
                        label = label.merge(temp, on='UID', how='left')
                        del temp
                        temp = \
                            op[['UID', deliver]].merge(op.groupby([deliver])[feature].nunique().reset_index(),
                                                       on=deliver,
                                                       how='left')[['UID', feature]]
                        temp = temp.groupby('UID')[feature].sum().reset_index()
                        temp.columns = ['UID', feature + deliver]
                        label = label.merge(temp, on='UID', how='left')
                        del temp
                        temp = \
                            op[['UID', deliver]].merge(op.groupby([deliver])[feature].max().reset_index(), on=deliver,
                                                       how='left')[['UID', feature]]
                        temp = temp.groupby('UID')[feature].mean().reset_index()
                        temp.columns = ['UID', feature + deliver]
                        label = label.merge(temp, on='UID', how='left')
                        del temp
                        temp = \
                            op[['UID', deliver]].merge(op.groupby([deliver])[feature].min().reset_index(), on=deliver,
                                                       how='left')[['UID', feature]]
                        temp = temp.groupby('UID')[feature].mean().reset_index()
                        temp.columns = ['UID', feature + deliver]
                        label = label.merge(temp, on='UID', how='left')
                        del temp
                        temp = \
                            op[['UID', deliver]].merge(op.groupby([deliver])[feature].sum().reset_index(), on=deliver,
                                                       how='left')[['UID', feature]]
                        temp = temp.groupby('UID')[feature].mean().reset_index()
                        temp.columns = ['UID', feature + deliver]
                        label = label.merge(temp, on='UID', how='left')
                        del temp
                        temp = \
                            op[['UID', deliver]].merge(op.groupby([deliver])[feature].mean().reset_index(), on=deliver,
                                                       how='left')[['UID', feature]]
                        temp = temp.groupby('UID')[feature].mean().reset_index()
                        temp.columns = ['UID', feature + deliver]
                        label = label.merge(temp, on='UID', how='left')
                        del temp
                        temp = \
                            op[['UID', deliver]].merge(op.groupby([deliver])[feature].std().reset_index(), on=deliver,
                                                       how='left')[['UID', feature]]
                        temp = temp.groupby('UID')[feature].mean().reset_index()
                        temp.columns = ['UID', feature + deliver]
                        label = label.merge(temp, on='UID', how='left')
                        del temp

        for feature in trans.columns[1:]:
            print(feature)
            if feature not in ['trans_amt', 'bal', 'day']:
                if feature != 'UID':
                    label = label.merge(trans.groupby(['UID'])[feature].count().reset_index(), on='UID', how='left')
                    label = label.merge(trans.groupby(['UID'])[feature].nunique().reset_index(), on='UID', how='left')
                for deliver in ['merchant', 'ip1', 'mac1', 'geo_code', ]:
                    if feature not in deliver:
                        if feature != 'UID':
                            temp = \
                                trans[['UID', deliver]].merge(trans.groupby([deliver])[feature].count().reset_index(),
                                                              on=deliver, how='left')[['UID', feature]]
                            temp = temp.groupby('UID')[feature].sum().reset_index()
                            temp.columns = ['UID', feature + deliver]
                            label = label.merge(temp, on='UID', how='left')
                            del temp
                            temp = \
                                trans[['UID', deliver]].merge(trans.groupby([deliver])[feature].nunique().reset_index(),
                                                              on=deliver, how='left')[['UID', feature]]
                            temp = temp.groupby('UID')[feature].sum().reset_index()
                            temp.columns = ['UID', feature + deliver]
                            label = label.merge(temp, on='UID', how='left')
                            del temp
                        else:
                            temp = \
                                trans[['UID', deliver]].merge(trans.groupby([deliver])[feature].count().reset_index(),
                                                              on=deliver, how='left')[['UID_x', 'UID_y']]
                            temp = temp.groupby('UID_x')['UID_y'].sum().reset_index()
                            temp.columns = ['UID', feature + deliver]
                            label = label.merge(temp, on='UID', how='left')
                            del temp
                            temp = \
                                trans[['UID', deliver]].merge(trans.groupby([deliver])[feature].nunique().reset_index(),
                                                              on=deliver, how='left')[['UID_x', 'UID_y']]
                            temp = temp.groupby('UID_x')['UID_y'].sum().reset_index()
                            temp.columns = ['UID', feature + deliver]
                            label = label.merge(temp, on='UID', how='left')
                            del temp
                if feature in ['merchant', 'code2', 'acc_id1', 'market_code', 'market_code']:
                    label[feature + '_z'] = 0
                    label[feature + '_z'] = label[feature + '_y'] / label[feature + '_x']
            else:
                print(feature)
                label = label.merge(trans.groupby(['UID'])[feature].count().reset_index(), on='UID', how='left')
                label = label.merge(trans.groupby(['UID'])[feature].nunique().reset_index(), on='UID', how='left')
                label = label.merge(trans.groupby(['UID'])[feature].max().reset_index(), on='UID', how='left')
                label = label.merge(trans.groupby(['UID'])[feature].min().reset_index(), on='UID', how='left')
                label = label.merge(trans.groupby(['UID'])[feature].sum().reset_index(), on='UID', how='left')
                label = label.merge(trans.groupby(['UID'])[feature].mean().reset_index(), on='UID', how='left')
                label = label.merge(trans.groupby(['UID'])[feature].std().reset_index(), on='UID', how='left')
                for deliver in ['merchant', 'ip1', 'mac1']:
                    if feature not in deliver:
                        temp = \
                            trans[['UID', deliver]].merge(trans.groupby([deliver])[feature].count().reset_index(),
                                                          on=deliver,
                                                          how='left')[['UID', feature]]
                        temp = temp.groupby('UID')[feature].sum().reset_index()
                        temp.columns = ['UID', feature + deliver]
                        label = label.merge(temp, on='UID', how='left')
                        del temp
                        temp = \
                            trans[['UID', deliver]].merge(trans.groupby([deliver])[feature].nunique().reset_index(),
                                                          on=deliver,
                                                          how='left')[['UID', feature]]
                        temp = temp.groupby('UID')[feature].sum().reset_index()
                        temp.columns = ['UID', feature + deliver]
                        label = label.merge(temp, on='UID', how='left')
                        del temp
                        temp = \
                            trans[['UID', deliver]].merge(trans.groupby([deliver])[feature].max().reset_index(),
                                                          on=deliver,
                                                          how='left')[['UID', feature]]
                        temp = temp.groupby('UID')[feature].mean().reset_index()
                        temp.columns = ['UID', feature + deliver]
                        label = label.merge(temp, on='UID', how='left')
                        del temp
                        temp = \
                            trans[['UID', deliver]].merge(trans.groupby([deliver])[feature].min().reset_index(),
                                                          on=deliver,
                                                          how='left')[['UID', feature]]
                        temp = temp.groupby('UID')[feature].mean().reset_index()
                        temp.columns = ['UID', feature + deliver]
                        label = label.merge(temp, on='UID', how='left')
                        del temp
                        temp = \
                            trans[['UID', deliver]].merge(trans.groupby([deliver])[feature].sum().reset_index(),
                                                          on=deliver,
                                                          how='left')[['UID', feature]]
                        temp = temp.groupby('UID')[feature].mean().reset_index()
                        temp.columns = ['UID', feature + deliver]
                        label = label.merge(temp, on='UID', how='left')
                        del temp
                        temp = \
                            trans[['UID', deliver]].merge(trans.groupby([deliver])[feature].mean().reset_index(),
                                                          on=deliver,
                                                          how='left')[['UID', feature]]
                        temp = temp.groupby('UID')[feature].mean().reset_index()
                        temp.columns = ['UID', feature + deliver]
                        label = label.merge(temp, on='UID', how='left')
                        del temp
                        temp = \
                            trans[['UID', deliver]].merge(trans.groupby([deliver])[feature].std().reset_index(),
                                                          on=deliver,
                                                          how='left')[['UID', feature]]
                        temp = temp.groupby('UID')[feature].mean().reset_index()
                        temp.columns = ['UID', feature + deliver]
                        label = label.merge(temp, on='UID', how='left')
                        del temp
        print("Done")
    except:
        pass
    label.columns = [str(i) + item if item not in tag_header else item for i, item in
                     enumerate(list(label.columns.values))]
    return label


def get_ftrs():
    # train = pd.DataFrame()
    # test = pd.DataFrame()
    # train, test = get_is_merchant_ftr(trans_train, trans_test, sub, y, train, test)
    print("features start!!")
    train = get_feature(op_train, trans_train, y)
    test = get_feature(op_test, trans_test, sub)
    train = fill_mean(train)
    test = fill_mean(test)
    print(train.shape)
    print(test.shape)
    # train, test = deal_data_v1(train, test)
    print(train.shape)
    # tr_corr = train.corr()[tag_hd.Tag].reset_index()
    # # tr_corr
    print(test.shape)
    # tr_corr = tr_corr[['Tag','index']].sort_values(by=['Tag'], ascending=True)
    # print(tr_corr)
    # col = tr_corr[tr_corr['Tag'].abs()>=0.1]['index']
    train.to_csv(features_base_path + "train_media_ftr.csv", index=False)
    test.to_csv(features_base_path + "test_media_ftr.csv", index=False)
    # train = get_train()
    # test = get_test()
    # test01[tag_hd.UID] = sub[tag_hd.UID]
    # train = train01.merge(train, on='UID', how='left')
    # test = test01.merge(test, on='UID', how='left')

get_ftrs()
